var path = require('path'), request = require('request'), cheerio = require('cheerio');
var Crawler = require('../../lib/crawler');
//var connection = 'mongodb://localhost:27017/Communities';
//var connection = 'mongodb://192.168.1.123:27017/66xiaoqu';
var connection = 'mongodb://120.55.126.201:27017/66xiaoqu';
var async = require('async');
var MongoClient = require('mongodb').MongoClient, assert = require('assert');
var ObjectId = require('mongodb').ObjectId

if (!String.prototype.format) {
    String.prototype.format = function () {
        var args = arguments;
        return this.replace(/{(\d+)}/g, function (match, number) {
            return typeof args[number] != 'undefined'
                ? args[number]
                : match
                ;
        });
    };
}

var dbconn = null;

function baiduRoads(filters, callbackOuter) {
    // tag is '交通设施'
    var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&region={1}&tag=%E4%BA%A4%E9%80%9A%E8%AE%BE%E6%96%BD&output=json&ak=AvRqTglpaKICTZ5iwyn06KZj&page_size=20&page_num='
    //var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&region={1}&tag=%E4%BA%A4%E9%80%9A%E8%AE%BE%E6%96%BD&output=json&ak=doqfhHvA5GjNLYnjjVFm9sL8&page_size=20&page_num='
    //var urlTemp = 'http://api.map.baidu.com/place/v2/search?&query={0}&region={1}&tag=%E4%BA%A4%E9%80%9A%E8%AE%BE%E6%96%BD&output=json&ak=ayGOiNeQNq6E9vSrbgeTrWbS&page_size=20&page_num='

    var sourceColl = 'baidu_md_roads';
    var targetColl = 'baidu_md_roadsNEW'
    var s_collection = null;
    var t_collection = null;

    function getRoads(callback) {
        MongoClient.connect(connection,
            function (err, db) {
                if (!err) {
                    dbconn = db
                    assert.equal(null, err);
                    console.log("Connected correctly to server");
                    t_collection = db.collection(targetColl);
                    s_collection = db.collection(sourceColl);
                    s_collection.count(filters, function (err, count) {
                        //totalData = count
                        //console.log("Total data %s: %s", dbCollection, totalData)
                        if (count == 0) {
                            console.log("NO DATA IN " + sourceColl + " FOR", filters)
                            dbconn.close()
                            callbackOuter()
                        }
                        else {
                            console.log("TOTAL DATA IN %s: %s", sourceColl, count)
                            callback()
                        }
                    });
                } else {
                    console.log(err)
                    callbackOuter()
                }
            });
    };

    function run() {
        var manifest = {
            target: {
                url: urlTemp,
                ranges: {
                    resource: 'mongodb' // configure it and put {{resource:mongodb}}
                },
                interval: 100 // Wait 0.5 seconds between each access
            },
            extractors: [{
                name: 'details',
                root: '',
                fields: {}
            }]
        };

        var job = new Crawler(manifest);

        job.on('data', function (data) {
        });

        job.on('error', function (err) {
            throw error;
        });

        job.on('end', function () {
        });

        function init(callback) {
            callback(null, 'init');
        }

        var items = [];

        function findCommids(db, callback) {
            s_collection.find(filters).toArray(
                function (err, docs) {
                    callback(docs);
                });
        };

        function getIds(callback) {
            findCommids(dbconn, function (docs) {
                    items = docs;
                    callback(null, 'getIds');
                }
            );
        }

        function insertDocuments(doc, callback) {
            t_collection.updateOne({'county':doc.county,'road':doc.road}, doc, {'upsert':true},function (err, result) {
                if (!err) {
                    console.log("\tUpserted %s (%s) into collection %s", doc.road, doc.city + ' ' + doc.county, targetColl);
                    callback(true)
                } else {
                    console.log(err)
                    callback(false)
                }
            });
        };

        function start(callback) {
            console.log("starting...");
            var cnt = 0;

            async.eachSeries(items, function (item, callbackNextItem) {

                    var query = item.city + ' ' + item.county + ' ' + item.road + ' 路口'
                    var region = item.city + ' ' + item.county
                    console.log("\n*****STARTING %s*****", query)
                    var url = urlTemp.format(encodeURIComponent(query), encodeURIComponent((region)))

                    var pageNo = 1

                    var ex_roads = []

                    async.whilst(function () {
                        return pageNo > 0
                    }, function (next) {
                        manifest.target.url = url + pageNo
                        console.log(manifest.target.url)

                        var job1 = new Crawler(manifest);
                        //console.log(job1)

                        job1.on('data', function (data) {
                            //console.log(data)
                            var $ = cheerio.load(data.request.body);
                            try {
                                var dataRoot = JSON.parse($._root.children[0].data)
                            } catch (e) {
                                console.log(e)
                                callbackNextItem();
                                pageNo = -1
                            }
                            //console.log(dataRoot)
                            if (parseInt(dataRoot.status) == 302) {
                                console.log("YOU HAVE BEEN BLOCKED BY BAIDU!")
                                process.exit()
                            }
                            else if (parseInt(dataRoot.status) == 240) {
                                console.log("INVALID DEVELOPER KEY!")
                                process.exit()
                            }
                            else if (parseInt(dataRoot.status) == 210) {
                                console.log("IP NOT ALLOWED FOR THIS DEVELOPER KEY!")
                                process.exit()
                            }
                            var data = dataRoot.results

                            var dataLen = data.length
                            if (dataLen == 0) {  // no results left
                                console.log("\nNo results left for %s!", query)
                                callbackNextItem();
                                pageNo = -1
                            } else {
                                var counter = 0
                                var addedComms = 0  //
                                async.eachSeries(data, function (doc, callbackNextDoc) {
                                    function goNext() {
                                        counter++
                                        if (counter == dataLen) {
                                            console.log("Extracted %s unique roads from %s page %s!", addedComms, query, pageNo)
                                            if (addedComms == 0 || dataLen < 20) {
                                                console.log("\nNo results left for %s!", query)
                                                callbackNextItem();  // important to end this loop BEFORE calling the next doc via callbackNextDoc()
                                                pageNo = -1;
                                            } else {
                                                pageNo++
                                                next()
                                            }
                                        }
                                        callbackNextDoc()
                                    }

                                    var name = doc.name
                                    if (name.indexOf('/') > -1) {
                                        var roads = name.split('/')
                                        var road1 = roads[0]
                                        var road2 = roads[1].replace('(路口)', '')
                                        var newdocs = []
                                        if (ex_roads.indexOf(road1) == -1) {
                                            newdocs.push({'road': road1})
                                        }
                                        if (ex_roads.indexOf(road2) == -1) {
                                            newdocs.push({'road': road2})
                                        }
                                        if (newdocs.length == 0) {
                                            goNext()
                                        } else {
                                            var innerCount = 0
                                            async.eachSeries(newdocs, function (new_doc, callbackInner3) {
                                                new_doc.city = item.city
                                                new_doc.cityId = item.cityId
                                                new_doc.county = item.county
                                                new_doc.countyId = item.countyId
                                                new_doc.sourceRoad = item.road

                                                new_doc.longitude = doc.location.lng
                                                new_doc.latitude = doc.location.lat
                                                insertDocuments(new_doc, function (success) {
                                                    if (success) {
                                                        ex_roads.push(new_doc.road)
                                                        addedComms++
                                                    }
                                                    innerCount++
                                                    if (innerCount == newdocs.length) {
                                                        goNext()
                                                    }
                                                    callbackInner3()

                                                })
                                            })
                                        }
                                    } else {
                                        //console.log("SKIPPED")
                                        goNext()
                                    }
                                })
                            }
                        });

                        job1.on('error', function (err) {
                            throw err;
                        });
                        job1.on('end', function () {
                            //console.log('The job is done: %d', pageNo);
                            delete job1;
                        });
                        job1.start();
                    }, function (err) {
                    })

                }, function (err) {
                    if (err) {
                        console.log('err');
                    }
                }
            );
        }

        function finish(callback) {
            //console.log("closing db...");
            callback(null, 'end');
        }

        // Start our job
        async.series([
                init,
                getIds,
                start,
                finish
            ]
            , function (err, results) {
                console.log(results);
            });
    }

    getRoads(run);
}

//var filters = {'county':'黄浦区'}
//var filters = {"_id": {$in: [ObjectId("567a40e5b802d011bf0f9f44"), ObjectId("567a40e5b802d011bf0f9f51"),ObjectId("567a40e5b802d011bf0f9f4c"),ObjectId("567a40e5b802d011bf0f9f43")]}}
var filters = {city:'北京'}
baiduRoads(filters, function () {
})

var exports = module.exports
exports.baiduRoads = baiduRoads